package each_webcode

import common.HbaseTools
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable
import scala.util.parsing.json.JSON;
object makeAnalysis {
        //输入文件idea相对路径:sub_EUROSTAT.txt
                //这个处理的应该是测试数据,实际数据应该是hdfs:/tmp/InternationalData_source/EUROSTAT_Value.00至06
                //代码作用:读取josn串,每一行返回用_拼接的key
                //输出文件:hdfs://node01:8020//tmp/InternationalData_tmp/sub_eurostat_test_46
                def main(args: Array[String]): Unit = {
                val sparkConf = new SparkConf().setAppName("FAOdatachange").setMaster("local[*]")
                val sc = new SparkContext(sparkConf)

                val inpath = "D:\\InternationalData\\makeChange\\src\\main\\resources\\sub_EUROSTAT.txt"
                val outpath = "hdfs://node01:8020//tmp/InternationalData_tmp/sub_eurostat_test_46"

                val webcodes = List("EUROSTAT")
                //make a cache:
                        val inputFile = inpath //读取json文件
                val jsonStr = sc.textFile(inputFile) //.sample(true,0.01);

                val result = jsonStr.map(s => JSON.parseFull(s))

                val rdds = result.map {
                        r =>
                                r match {
                                        case Some(maps: Map[String, Any]) => {
                                                val webcode = "EUROSTAT"
                                                val thashmap: mutable.HashMap[String, String] = new mutable.HashMap[String, String]()
                                                maps.foreach(t => {
                                                        thashmap.put(t._1.toString , t._2.toString)
                                                        })
                                                thashmap.keys.toList.mkString("_")
                                                }
                                        }
                        }.distinct.saveAsTextFile(outpath)
                }
        }
